{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import scipy.special\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NeuralNetwork:\n",
    "    # initialise the neural network\n",
    "    def __init__(self, input_nodes, hidden_nodes, output_nodes, learning_rate) -> None:\n",
    "        self.inodes = input_nodes\n",
    "        self.hnodes = hidden_nodes\n",
    "        self.onodes = output_nodes\n",
    "        self.lr = learning_rate\n",
    "        self.correct_rate = 0.0\n",
    "        self.label_scores = {}\n",
    "\n",
    "        # set weight: input -> hidden, -0.5 ~ +0.5\n",
    "        # self.wih = np.random.rand(self.hnodes, self.inodes) - 0.5\n",
    "        self.wih = np.random.normal(0.0, pow(self.hnodes, -0.5), (self.hnodes, self.inodes))\n",
    "        # hidden -> output\n",
    "        # self.who = np.random.rand(self.onode, self.hnodes) - 0.5\n",
    "        self.who = np.random.normal(0.0, pow(self.onodes, -0.5), (self.onodes, self.hnodes))\n",
    "\n",
    "        # activation function: sigmoid = 1/(1+exp(-x))\n",
    "        self.activation_function = lambda x: scipy.special.expit(x)\n",
    "\n",
    "        # activation funciton for back query: x = ln(y/(1-y))\n",
    "        self.back_activation_func = lambda y: scipy.special.logit(y)\n",
    "\n",
    "    # train the neural network\n",
    "    def train(self, input_list, target_list):\n",
    "        inputs = np.array(input_list, ndmin=2).T\n",
    "        targets = np.array(target_list, ndmin=2).T\n",
    "        hidden_inputs = np.dot(self.wih, inputs)\n",
    "        hidden_outputs = self.activation_function(hidden_inputs)\n",
    "        final_inputs = np.dot(self.who, hidden_outputs)\n",
    "        final_outputs = self.activation_function(final_inputs)\n",
    "\n",
    "        # calculate errors\n",
    "        output_errors = targets - final_outputs\n",
    "        hidden_errors = np.dot(self.who.T, output_errors)\n",
    "\n",
    "        # update weights\n",
    "        self.who += self.lr * np.dot(output_errors * final_outputs * (1.0 - final_outputs), np.transpose(hidden_outputs))\n",
    "        self.wih += self.lr * np.dot(hidden_errors * hidden_outputs * (1.0 - hidden_outputs), np.transpose(inputs))\n",
    "\n",
    "    # query the neural network\n",
    "    def query(self, input_list):\n",
    "        # convert input to 2d array\n",
    "        inputs = np.array(input_list, ndmin=2).T\n",
    "        hidden_inputs = np.dot(self.wih, inputs)\n",
    "        hidden_outputs = self.activation_function(hidden_inputs)\n",
    "        final_inputs = np.dot(self.who, hidden_outputs)\n",
    "        final_outputs = self.activation_function(final_inputs)\n",
    "        return final_outputs\n",
    "\n",
    "    # back query, input label, output image\n",
    "    def back_query(self, label):\n",
    "        final_outputs = np.zeros(self.onodes) + 0.01\n",
    "        final_outputs[label] = 0.99\n",
    "\n",
    "        final_inputs = self.back_activation_func(final_outputs)\n",
    "\n",
    "        hidden_outputs = np.dot(self.who.T, final_inputs) # ?\n",
    "        # scale outputs to 0.01 ~ 0.99, as the sigmoid value region is (0, 1).\n",
    "        # and here we just handle all inputs values without excluding.\n",
    "        # so the scale is required to make sure all input for sigmoid is valid.\n",
    "        hidden_outputs -= np.min(hidden_outputs)\n",
    "        hidden_outputs /= np.max(hidden_outputs)\n",
    "        hidden_outputs *= 0.98\n",
    "        hidden_outputs += 0.01\n",
    "\n",
    "        hidden_inputs = self.back_activation_func(hidden_outputs)\n",
    "\n",
    "        inputs = np.dot(self.wih.T, hidden_inputs)\n",
    "        inputs -= np.min(inputs)\n",
    "        inputs /= np.max(inputs)\n",
    "        inputs *= 0.98\n",
    "        inputs += 0.01\n",
    "\n",
    "        return inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_data_file = open('mnist_dataset/mnist_train.csv', 'r')\n",
    "training_data_list = training_data_file.readlines()\n",
    "training_data_file.close()\n",
    "# test results\n",
    "test_data_file = open('mnist_dataset/mnist_test.csv', 'r')\n",
    "test_data_list = test_data_file.readlines()\n",
    "test_data_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def do_train_and_test(model, output_nodes, learning_rate, correct_rates, epochs):\n",
    "    for epoch in range(epochs):\n",
    "        for record in training_data_list:\n",
    "            all_values = record.split(',')\n",
    "            inputs = np.asfarray(all_values[1:]) / 255.0 * 0.99 + 0.01\n",
    "            targets = np.zeros(output_nodes) + 0.01\n",
    "            targets[int(all_values[0])] = 0.99\n",
    "            model.train(inputs, targets)\n",
    "\n",
    "    # record scores\n",
    "    score_card = []\n",
    "    num_scores = {0:0,1:0,2:0,3:0,4:0,5:0,6:0,7:0,8:0,9:0}\n",
    "    num_total = {0:0,1:0,2:0,3:0,4:0,5:0,6:0,7:0,8:0,9:0}\n",
    "    for record in test_data_list:\n",
    "        all_values = record.split(',')\n",
    "        correct_label = int(all_values[0])\n",
    "        num_total[correct_label] += 1\n",
    "        inputs = np.asfarray(all_values[1:]) / 255.0 * 0.99 + 0.01\n",
    "        outputs = model.query(inputs)\n",
    "        # get index of max value \n",
    "        label = np.argmax(outputs)\n",
    "        if label == correct_label:\n",
    "            score_card.append(1)\n",
    "            num_scores[label] += 1\n",
    "        else:\n",
    "            score_card.append(0)\n",
    "    score_card_array = np.asarray(score_card)\n",
    "    model.correct_rate = score_card_array.sum() / score_card_array.size\n",
    "    correct_rates[learning_rate] = model.correct_rate\n",
    "    for key in num_total:\n",
    "        if num_total[key] != 0:\n",
    "            num_scores[key] /= num_total[key]\n",
    "    model.label_scores = num_scores\n",
    "    return num_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: 0.9897959183673469,\n",
       " 1: 0.986784140969163,\n",
       " 2: 0.939922480620155,\n",
       " 3: 0.9623762376237623,\n",
       " 4: 0.9511201629327902,\n",
       " 5: 0.9428251121076233,\n",
       " 6: 0.9561586638830898,\n",
       " 7: 0.9426070038910506,\n",
       " 8: 0.9527720739219713,\n",
       " 9: 0.9504459861248761}"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# cost about 1 min 5 secs\n",
    "epochs = 2\n",
    "learning_rate = 0.2\n",
    "input_nodes = 784 # image is 28 x 28\n",
    "hidden_nodes = 100\n",
    "output_nodes = 10 # 0 ~ 9 total 10 labels\n",
    "correct_rates = {}\n",
    "n = NeuralNetwork(input_nodes, hidden_nodes, output_nodes, learning_rate)\n",
    "num_score = do_train_and_test(n, output_nodes, learning_rate, correct_rates, epochs)\n",
    "num_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0.2: 0.9579}"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "correct_rates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "# recogonize customized numbers\n",
    "import cv2\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x2572c3547f0>"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAOs0lEQVR4nO3df4gcdZrH8c9zcxFCDNFxxhiymuRUEDm9MQ7DwspGWRQNSLIqskEWD4JZQWGX5A9/BFxRgiKXXRY9lESjWbOXuP72D3PuGBfiIqwZ45xJlJxZTVzj6HSMGKOJuUye+2Mqy6hT357pru5q53m/YOjuerpSD4Ufq7u+Vf01dxeAie+fym4AQHMQdiAIwg4EQdiBIAg7EMQ/N3NjHR0dPnv27GZuEghl9+7d2rdvn41WqyvsZna5pN9JapP0sLvfm3r/7Nmz1dfXV88mASR0d3fn1mr+GG9mbZL+U9IVks6VtMjMzq313wPQWPV8Z++RtMvd33P3I5I2SFpQTFsAilZP2GdK+vuI1x9my77BzJaYWZ+Z9VUqlTo2B6AeDT8b7+6r3L3b3bs7OzsbvTkAOeoJ+15Jp494/YNsGYAWVE/Yt0g628zmmNkJkn4m6YVi2gJQtJqH3tz9qJndLOklDQ+9rXH3HYV1BqBQdY2zu/uLkl4sqBcADcTlskAQhB0IgrADQRB2IAjCDgRB2IEgmno/e1R796YvLLznnnuS9UcffTRZP/HEE3NrM2d+53aFbzjrrLOS9csuuyxZv+qqq5L19vb2ZB3Nw5EdCIKwA0EQdiAIwg4EQdiBIAg7EARDb01w3333JesPPvhgsn7s2LFk/auvvsqtDQ4OJtd98803k/WnnnoqWV+/fn2y/swzz+TWpk2bllwXxeLIDgRB2IEgCDsQBGEHgiDsQBCEHQiCsANBMM7eBOvWrUvWq42jl8ndk/VXXnklWb/yyitza88991xyXW6PLRZHdiAIwg4EQdiBIAg7EARhB4Ig7EAQhB0IgnH2JjjppJOS9f379zenkRK8+uqrubXHHnssue7SpUsL7ia2usJuZrslfSFpSNJRd+8uoikAxSviyH6Ju+8r4N8B0EB8ZweCqDfsLulPZvaGmS0Z7Q1mtsTM+sysr1Kp1Lk5ALWqN+wXuftcSVdIusnMfvztN7j7Knfvdvfuzs7OOjcHoFZ1hd3d92aPg5KeldRTRFMAildz2M1siplNPf5c0mWSthfVGIBi1XM2frqkZ83s+L/zX+7+34V0NcFU+934G264IVn/7LPPimynZTzwwAPJOuPsxao57O7+nqR/K7AXAA3E0BsQBGEHgiDsQBCEHQiCsANBcItrE8yfPz9Z37RpU7L+0ksvJes7duzIrW3cuDG57qeffpqsN9L777+frA8NDSXrbW1tRbYz4XFkB4Ig7EAQhB0IgrADQRB2IAjCDgRB2IEgGGdvgsmTJyfrF1xwQbJ+/vnnJ+tffvllbq23tze57jXXXJOsl+nw4cPJ+pQpU5rUycTAkR0IgrADQRB2IAjCDgRB2IEgCDsQBGEHgmCcvQCHDh1K1jdv3pys33333cn666+/nqwfPXo0t+buyXVb2ddff52sM84+PhzZgSAIOxAEYQeCIOxAEIQdCIKwA0EQdiAIxtkL0N/fn6wvWrQoWZ+oUzLXa9asWcn6woULk/Ubb7wxtzZ37tzkutV+g+D7qOqR3czWmNmgmW0fsazdzHrN7N3s8eTGtgmgXmP5GP+YpMu/texWSZvc/WxJm7LXAFpY1bC7+2ZJ+7+1eIGktdnztZIWFtsWgKLVeoJuursPZM8/ljQ9741mtsTM+sysr1Kp1Lg5APWq+2y8D99pkXu3hbuvcvdud+/u7Oysd3MAalRr2D8xsxmSlD0OFtcSgEaoNewvSLo+e369pOeLaQdAo1QdZzez9ZIultRhZh9K+rWkeyX90cwWS9oj6dpGNtnqVq5cmawzjl6bgwcPJuvr1q1L1jds2JBbW758eXLdW265JVn/Po7DVw27u+ddEfKTgnsB0EBcLgsEQdiBIAg7EARhB4Ig7EAQ3OKKCSv1E9t33XVXct2Ojo5kffHixcl6Kw7NcWQHgiDsQBCEHQiCsANBEHYgCMIOBEHYgSAYZy/AsmXLkvUtW7Yk6x988EGR7WAMqk1lfdtttyXr5513XrI+b968cffUaBzZgSAIOxAEYQeCIOxAEIQdCIKwA0EQdiAIxtkL0NXVlaw/+eSTyfrTTz+drO/cuTNZP3z4cG6t2n3VkyZNStY/+uijZH3Xrl3J+oEDB3Jrhw4dSq5bpmo/Y71ixYpknXF2AKUh7EAQhB0IgrADQRB2IAjCDgRB2IEgGGcvQLWx7J6enrrq32dDQ0O5tWr38Ve7p/yJJ56oqaci9Pb2lrbtWlU9spvZGjMbNLPtI5bdaWZ7zaw/+5vf2DYB1GssH+Mfk3T5KMt/6+5d2d+LxbYFoGhVw+7umyXtb0IvABqonhN0N5vZW9nH/JPz3mRmS8ysz8z6KpVKHZsDUI9aw/6gpDMldUkakLQy743uvsrdu929u7Ozs8bNAahXTWF390/cfcjdj0laLWnink4GJoiawm5mM0a8/Kmk7XnvBdAaqo6zm9l6SRdL6jCzDyX9WtLFZtYlySXtlvSLxrWI77O2trbc2pw5c5LrVptDvcxx9hNOOKG0bdeqatjdfdEoix9pQC8AGojLZYEgCDsQBGEHgiDsQBCEHQiCW1zRsrZt21Z2C7mOHDlSdgvjxpEdCIKwA0EQdiAIwg4EQdiBIAg7EARhB4JgnB2l+fzzz5P1pUuXNqmT8bv00kvLbmHcOLIDQRB2IAjCDgRB2IEgCDsQBGEHgiDsQBCMs6OhDh06lFu74447kutWm9K5TMuXLy+7hXHjyA4EQdiBIAg7EARhB4Ig7EAQhB0IgrADQTDOjrpUuyd9xYoVubX777+/6HYKU+1+9Z6eniZ1UpyqR3YzO93M/mxmb5vZDjP7Zba83cx6zezd7PHkxrcLoFZj+Rh/VNIydz9X0g8l3WRm50q6VdImdz9b0qbsNYAWVTXs7j7g7luz519IekfSTEkLJK3N3rZW0sIG9QigAOM6QWdmsyVdIOmvkqa7+0BW+ljS9Jx1lphZn5n1VSqVenoFUIcxh93MTpT0tKRfufuBkTV3d0k+2nruvsrdu929u7Ozs65mAdRuTGE3s0kaDvof3P2ZbPEnZjYjq8+QNNiYFgEUoerQm5mZpEckvePuvxlRekHS9ZLuzR6fb0iHqGpoaCi3tn///uS6W7duTdY3btyYrK9fvz5ZHxxszWNAe3t7sv7QQw8l65MnTy6ynaYYyzj7jyT9XNI2M+vPlt2u4ZD/0cwWS9oj6dqGdAigEFXD7u5/kWQ55Z8U2w6ARuFyWSAIwg4EQdiBIAg7EARhB4LgFtcxSv0kcrWx6tWrVyfrL7/8crJ+8ODBZD3V25EjR5LrTmTTpk3Lra1Zsya57qxZs4pup3Qc2YEgCDsQBGEHgiDsQBCEHQiCsANBEHYgCMbZx6i/vz+3dt111yXX3bNnT8HdQKp+T/rjjz+eW7vkkkuS67a1tdXUUyvjyA4EQdiBIAg7EARhB4Ig7EAQhB0IgrADQTDOPkYrV67MrTGO3hhnnnlmst7b25usn3HGGbm1iTiOXg1HdiAIwg4EQdiBIAg7EARhB4Ig7EAQhB0IYizzs58u6feSpktySavc/XdmdqekGyRVsrfe7u4vNqpRTDxXX311sl5tjvSOjo4i25nwxnJRzVFJy9x9q5lNlfSGmR2/muG37v4fjWsPQFHGMj/7gKSB7PkXZvaOpJmNbgxAscb1nd3MZku6QNJfs0U3m9lbZrbGzE7OWWeJmfWZWV+lUhntLQCaYMxhN7MTJT0t6VfufkDSg5LOlNSl4SP/qBePu/sqd+929+7Ozs76OwZQkzGF3cwmaTjof3D3ZyTJ3T9x9yF3PyZptaSexrUJoF5Vw25mJukRSe+4+29GLJ8x4m0/lbS9+PYAFGUsZ+N/JOnnkraZWX+27HZJi8ysS8PDcbsl/aIB/bWMZcuW5dYGBgaS67722mtFt1OYqVOnJusXXnhhsj5v3rxkfcGCBbm1c845J7nu5MmTk3WMz1jOxv9Fko1SYkwd+B7hCjogCMIOBEHYgSAIOxAEYQeCIOxAEPyU9Bh1dXXl1h5++OHkujt37kzWh69bynfaaacl66eeempu7ZRTTkmuO2XKlGQ94k8uT1Qc2YEgCDsQBGEHgiDsQBCEHQiCsANBEHYgCHP35m3MrCJp5PzGHZL2Na2B8WnV3lq1L4nealVkb7PcfdTff2tq2L+zcbM+d+8urYGEVu2tVfuS6K1WzeqNj/FAEIQdCKLssK8qefsprdpbq/Yl0VutmtJbqd/ZATRP2Ud2AE1C2IEgSgm7mV1uZjvNbJeZ3VpGD3nMbLeZbTOzfjPrK7mXNWY2aGbbRyxrN7NeM3s3exx1jr2ServTzPZm+67fzOaX1NvpZvZnM3vbzHaY2S+z5aXuu0RfTdlvTf/ObmZtkv5X0qWSPpS0RdIid3+7qY3kMLPdkrrdvfQLMMzsx5IOSvq9u/9rtuw+Sfvd/d7sf5Qnu/stLdLbnZIOlj2NdzZb0YyR04xLWijp31Xivkv0da2asN/KOLL3SNrl7u+5+xFJGyTlTxsSmLtvlrT/W4sXSFqbPV+r4f9Ymi6nt5bg7gPuvjV7/oWk49OMl7rvEn01RRlhnynp7yNef6jWmu/dJf3JzN4wsyVlNzOK6e5+fL6pjyVNL7OZUVSdxruZvjXNeMvsu1qmP68XJ+i+6yJ3nyvpCkk3ZR9XW5IPfwdrpbHTMU3j3SyjTDP+D2Xuu1qnP69XGWHfK+n0Ea9/kC1rCe6+N3sclPSsWm8q6k+Oz6CbPQ6W3M8/tNI03qNNM64W2HdlTn9eRti3SDrbzOaY2QmSfibphRL6+A4zm5KdOJGZTZF0mVpvKuoXJF2fPb9e0vMl9vINrTKNd9404yp535U+/bm7N/1P0nwNn5H/m6TlZfSQ09e/SPqf7G9H2b1JWq/hj3X/p+FzG4slnSJpk6R3Jb0sqb2Fentc0jZJb2k4WDNK6u0iDX9Ef0tSf/Y3v+x9l+irKfuNy2WBIDhBBwRB2IEgCDsQBGEHgiDsQBCEHQiCsANB/D/UUXEigcyDdwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "img = cv2.imread('images/3-2.png', cv2.IMREAD_GRAYSCALE)\n",
    "img_data = (255.0 - img.reshape(28, 28)) / 255.0 * 0.99 + 0.01\n",
    "predicts = n.query(img_data.flatten())\n",
    "print(np.argmax(predicts))\n",
    "plt.imshow(img_data, cmap='Greys', interpolation='None')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x2573326fee0>"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAVtElEQVR4nO3dXWyc5ZUH8P9xbCfBdmISJ8ax07XzQUKAJICBlSALq2pRyA30BpWLipXQBokitVIvFrEX5RKttq16sarkLqjpiqWq1CK4iKAsqkQqQYUT5QvCRwgO2PFHEjuJnZA4ts9eeMIa8Hv+7rz2zIjn/5Mi23P8zjzzzpzMeM5znsfcHSLy7VdV7gGISGko2UUSoWQXSYSSXSQRSnaRRFSX8sbq6uq8sbExM15VFf/fMzU1VfRts2PNrOjrZhYtWhTG2djYeYnGnvd+5z0+ik9OTobHVlfHT8+JiYmij2fHsnPO5HlM2XmJzum5c+dw6dKlWX8hV7Kb2U4AvwSwCMB/uftz0e83NjbiiSeeyIxfd9114e198cUXmTH24Fy6dCmMsydWnge/rq4ujF+5ciWML168OIwvWbIkMzY2Npbrutl5Y+dl6dKlmbGRkZHw2JUrV4bx4eHhMN7U1JQZO3PmTHhsdE4Bfr9HR0fD+LJlyzJj586dC4+tqanJjHV1dWXGin4Gm9kiAP8J4EEAWwA8amZbir0+EVlYed6r3AXguLufcPdxAL8D8ND8DEtE5lueZG8F8PmMn3sLl32Fme02s24z67548WKOmxORPBb803h373L3TnfvZH+7isjCyZPsfQDWzvi5rXCZiFSgPMn+LoCNZtZhZrUAvg/g1fkZlojMt6JLb+4+YWZPAXgd06W3F9z9veiYqqqqsAx14cKF8Daj8tjVq1fDY1kpJU+9mdVUWTxvvTkqSbLSGbttVo9m5/XUqVOZsTVr1oTHsrEzUUmzoaEhPJY9ZmzuRDSfBIjPa319fXjs+Ph4GM+Sq87u7nsB7M1zHSJSGpouK5IIJbtIIpTsIolQsoskQskukgglu0giStrPPjk5ifPnz2fGWe0zWgmXzbtntUlWL45aNVmbaHSfAd4uefny5aKvn90vNj8hby9+1MrJ7hd7zPL00g8NDYXHsvkFbW1tYTya+wDEjwub+1BsnV2v7CKJULKLJELJLpIIJbtIIpTsIolQsoskoqSlNzMLV8bMU3Jgq+BEtwsAAwMDYby19Rsrbn2Jlf3Yqrm1tbVhPM9qo6dPnw6PZeUvVoJipbmoTXXt2rWZMYA/ZkzUXsvKvGxlW/aYsDbVqGTJzmlUzoyO1Su7SCKU7CKJULKLJELJLpIIJbtIIpTsIolQsoskoqR19qmpqbAmnadWznbNZHVV1rIYtdeyuihrSWQtsGyX16iVk9XR2fLdq1atCuOslTNqDT569Gh47Pr168N4Hqz1lz2fovsF8McsmnvBzmmUJ9FzQa/sIolQsoskQskukgglu0gilOwiiVCyiyRCyS6SiJLW2auqqsJ6N+v7jmr0bDlnhtU2o1r66tWrw2P7+/vDOKvJsq2LBwcHM2Psfu3bty+Ms7kPzc3NYXzz5s1hPPL666+H8R07doTxaI4AW4OAPSbsvLC5FxE2LyN6PkTrD+RKdjPrATAKYBLAhLt35rk+EVk48/HK/o/ufmYerkdEFpD+ZhdJRN5kdwB/MrP9ZrZ7tl8ws91m1m1m3ezvJBFZOHnfxt/r7n1mthrAG2b2gbu/NfMX3L0LQBcAtLa2ZneTiMiCyvXK7u59ha9DAF4GcNd8DEpE5l/RyW5mdWbWcO17AA8AiHsWRaRs8ryNbwbwcqF/thrA/7j7a+ygqC+c9V5H/eysnsxq+Gzb5Kh+efjw4fBYtnZ7R0dHGD9zJi523HDDDZmxvDX+FStWhPFNmzaF8WhsjY2N4bFNTU1hnO0zcPbs2czYkSNHwmPb29vD+LZt28L48uXLw3jUL8967aM5JdXV2SlddLK7+wkA8T0WkYqh0ptIIpTsIolQsoskQskukgglu0giSt7iGpXA2PK9USlmZGSE3nYex48fz4xF5USAb/8bbS0M8BbXqLwWbe8LADt37gzjbFvlkydPhvHo3IyNjYXHvvjii2GcLTXd0tKSGYuWXAb4lszR8wEAbr755jAetcCy8xK1wEblSL2yiyRCyS6SCCW7SCKU7CKJULKLJELJLpIIJbtIIkq+ZXNUM2btlNGSyVFrH8CXDo7aZ4F4i152LKubsmWH2X2LWmRZeyzz9ttvh3E2v+G117K7nrdv3x4ey84La8+NHjPWPltbWxvG2XLPV69eDePRMtds3kb0fIjmk+iVXSQRSnaRRCjZRRKhZBdJhJJdJBFKdpFEKNlFElHSOjsQ9xGzWnhUN82z3DIADA0NhfGo7sqWRGb96qxmy64/WnqY9W2zfnTW1x09JkBcS49qzUD+bbijWjcbN1ummi1Nzmrl0bLpbCnpaNn06Hb1yi6SCCW7SCKU7CKJULKLJELJLpIIJbtIIpTsIokoaZ190aJFaGhoyIyzbZej2uSaNWvCY9na66x3OtqyOe91s351dt8OHjyYGWPbYH/nO98J4/fcc08YZ2u3Rw4dOhTGWZ399ttvD+PRmvnsvLC5DfX19WGcrc3Q1taWGTt27Fh47IULFzJjudaNN7MXzGzIzI7OuGyFmb1hZh8Xvl7PrkdEymsub+N/A+Dr24Y8DeBNd98I4M3CzyJSwWiyu/tbAIa/dvFDAPYUvt8D4OH5HZaIzLdiP6BrdvdrG4wNAGjO+kUz221m3WbWzdZiE5GFk/vTeJ+eeZ85+97du9y909072YcaIrJwik32QTNrAYDC17hlTETKrthkfxXAY4XvHwPwyvwMR0QWCq2zm9lLAO4H0GRmvQB+CuA5AL83s8cBnATwyFxu7OrVq2HfOOs5j+rwrJbN+ovZOuGffvppZozt/c726mafZezfvz+MDw9//fPT/3fTTTeFx7Lz9uSTT4Zxdt+XL1+eGWPrF+zatSuMR/VmAOjp6cmMbdmyJTyWrfvO5oTkWQeA9bNH8weix5Mmu7s/mhH6LjtWRCqHpsuKJELJLpIIJbtIIpTsIolQsoskoqQtrtXV1WHrX19fX3j89ddnN9flXfL4nXfeCeOHDx/OjG3atCk8NioBAbz0Fm1VDcRlRdZ+u2PHjjDe3t4exlmZKCphsTZQ1ob60UcfhfGtW7dmxth5OXDgQBhnx994441h/MEHH8yMsaXFp6amwngWvbKLJELJLpIIJbtIIpTsIolQsoskQskukgglu0giSlpnd/dc2+iOjIxkxtiyw5999lkYZ+2S0RK9rN7LtpNm2/+yNtSopstaXDds2BDG2VbW0dwHIK6znz17NjyWzT/Ytm1bGN+8eXNmjG1VzeZ8sHkdd999dxg/f/58ZizPls1RDV6v7CKJULKLJELJLpIIJbtIIpTsIolQsoskQskukoiS1tnNLKyHs/7laGvj5ubMHagA8Jou60++cuVKZozVotlS0lENH+DLNUfHNzU1hceysa9evTqMsyW6ozp7tAQ2wLdNZvMyTp8+XfRtr1u3LoyzOjtbajqan8CW2I7ywMwyY3plF0mEkl0kEUp2kUQo2UUSoWQXSYSSXSQRSnaRRJS0zj45ORnWJ1kfb2R0dDSMs3pw1F8MAC0tLZkx1o/OrjuqjQK8n/3WW2/NjK1atSo89o477gjjNTU1YXxgYCCMR/MT2GPC4mxuRLSGAdsevK6uLoxHzwcAaGhoCOPRfWPzTdjzIQt9ZTezF8xsyMyOzrjsWTPrM7ODhX/xRtoiUnZzeRv/GwA7Z7n8F+6+vfBv7/wOS0TmG012d38LQDy3UEQqXp4P6J4ys8OFt/mZE33NbLeZdZtZN/sbS0QWTrHJ/isA6wFsB9AP4GdZv+juXe7e6e6d7IMsEVk4RSW7uw+6+6S7TwH4NYC75ndYIjLfikp2M5tZd/gegKNZvysilYHW2c3sJQD3A2gys14APwVwv5ltB+AAegA8MZcbM7Owbhv16QJx7ZP1D7NaNqvptrW1ZcbYuNm68nfeeWcYb21tDeO33XZbZoytrc7+tIrW6geA3t7eMP7JJ59kxth6+qxnnNXCo354tkYA25ee7ZHO9p7PM99kYmIijGehye7uj85y8fNF3ZqIlI2my4okQskukgglu0gilOwiiVCyiySipC2uVVVVYVmBLakclYFYGYaVWqJtj4F46V9Wfuro6Ajj7H5HSyIDcQstK9Ow0hprHT527FgYP3ToUGaMnTfWhvr555+H8aj0xlpQ2VLTy5cvD+N5lv9mx0ZxLSUtIkp2kVQo2UUSoWQXSYSSXSQRSnaRRCjZRRJR0jr7xMREWNdlrX1Re2xtbW14bLRVNBC3sAJAX19fZozVRaMaPcC36GX3Ldp2mW3JHG2pDAB798Zrie7bty+MRy2ubEtmtt30hQsXwni07fL4+Hh4LFtCm42dzfuIWovZvIuoxh8tM61XdpFEKNlFEqFkF0mEkl0kEUp2kUQo2UUSoWQXSUTJ+9mXLl2aGWe1zaiezbaxZTXbqI4OAIODg5kxVsNn/eis57ynpyeMRz788MMwvmHDhjDe3d0dxtkcgPvuuy8zxrayjmr0ALB58+YwHj2mrFeePZ/WrFkTxlkdP5pbEeUIAPT392fGonkTemUXSYSSXSQRSnaRRCjZRRKhZBdJhJJdJBFKdpFElLTO7u5h/ZHVNqM+X7ZtMtvSmW3ZvHXr1szYwMBAeCzrZ//ggw/C+OXLl8N4VIdncwD2798fxm+55ZYwznrxo95rto12S0tLGGdr2jc3N2fG2JbLGzduDONsbGwdgWjOSJ7nahSjr+xmttbM/mxm75vZe2b2o8LlK8zsDTP7uPA1fkaLSFnN5W38BICfuPsWAH8P4IdmtgXA0wDedPeNAN4s/CwiFYomu7v3u/uBwvejAI4BaAXwEIA9hV/bA+DhBRqjiMyDv+kDOjNrB3AbgL8CaHb3a5N0BwDM+geSme02s24z67506VKesYpIDnNOdjOrB/AHAD9296+s9OfTnwrM+smAu3e5e6e7d0aL7InIwppTsptZDaYT/UV3/2Ph4kEzaynEWwDEHz+KSFnR0ptN10eeB3DM3X8+I/QqgMcAPFf4+gq7rqqqqnCJXVb+itopoxbUuVw3K6VEpRq2dTArpbA20QceeCCMR/eNtXKyVswrV66E8fb29jA+NjaWGWNtx6xsuH79+jAetdCyx2T16tVh/OTJk2GcLS8elVPZMtRRKTcqQc+lzn4PgB8AOGJmBwuXPYPpJP+9mT0O4CSAR+ZwXSJSJjTZ3f0vALJmP3x3focjIgtF02VFEqFkF0mEkl0kEUp2kUQo2UUSUfIW12ipW9Z2GC2xG7VSAsDw8HAY7+joCOMnTpzIjLGa7Llz58L4qlWrwjirCUc13VOnToXHsm2P2X1j7b1RGytbMpnV8JctWxbGo+cEe7709vaGcbbVNRtb9JixLZujVvBcLa4i8u2gZBdJhJJdJBFKdpFEKNlFEqFkF0mEkl0kESWvs0f90ayvO6oJs7onq+kuWbIkjEfLErPbZnV0dr9ZzTeqpbPbZst3s75t1i8fLUXNlik7e/ZsGGdLbEe1dLb+ARsbe8zZ0ubR8Wzthei8RDV6vbKLJELJLpIIJbtIIpTsIolQsoskQskukgglu0giSlpnr66uDtcKZ2ttRzV6VpuM1i8HeH9zVAtfuXJleCzrT25sbAzjrBa+YcOGzBjrpWc1ftaXHW0XDQCnT5/OjLH10aNjAeDMmTNhvKamJjPW0NAQHsu2k2bY2KLHlO2ctGBbNovIt4OSXSQRSnaRRCjZRRKhZBdJhJJdJBFKdpFEzGV/9rUAfgugGYAD6HL3X5rZswD+BcC1Yugz7r43z2BYXTWqfbJ6MKvpXrx4MYxH9UvWKx/VewFe4x8ZGQnj0fyDdevWhceyvmy2Pzur00f6+/uLPhbga9pH8xvY3AW2Vj+rhbPnY9SLz54v0RoF0bFzmVQzAeAn7n7AzBoA7DezNwqxX7j7f8zhOkSkzOayP3s/gP7C96NmdgxA60IPTETm19/0N7uZtQO4DcBfCxc9ZWaHzewFM7s+45jdZtZtZt1syqqILJw5J7uZ1QP4A4Afu/sFAL8CsB7Adky/8v9stuPcvcvdO929s76+Pv+IRaQoc0p2M6vBdKK/6O5/BAB3H3T3SXefAvBrAHct3DBFJC+a7Db9EfjzAI65+89nXN4y49e+B+Do/A9PRObLXD6NvwfADwAcMbODhcueAfComW3HdDmuB8AT7IomJyfDlktW5onKX4sXL2Y3HxofHw/jExMTmTHWDpm3/ZaVgaLyWbTVNMBLZ6xkyZbgjkpUUbszwMuhrPwVld5Y2zFrWx4dHQ3jrFwalchYy3PUPhttez6XT+P/AmC2Z3OumrqIlJZm0IkkQskukgglu0gilOwiiVCyiyRCyS6SiJIuJV1VVRXWbVk9esWKFZkxVqtmtXB2fHt7e2aM1VRZLZu1mbJ6dJ5lj9nWxKzOzu5bNEWa1arZdtNsbkSEtZGy5wM7L6yOH7XAsrFF5zx6LuiVXSQRSnaRRCjZRRKhZBdJhJJdJBFKdpFEKNlFEmGstj2vN2Z2GsDJGRc1AYj3ti2fSh1bpY4L0NiKNZ9j+zt3n3WCQkmT/Rs3btbt7p1lG0CgUsdWqeMCNLZilWpsehsvkgglu0giyp3sXWW+/Uiljq1SxwVobMUqydjK+je7iJROuV/ZRaRElOwiiShLspvZTjP70MyOm9nT5RhDFjPrMbMjZnbQzLrLPJYXzGzIzI7OuGyFmb1hZh8Xvs66x16ZxvasmfUVzt1BM9tVprGtNbM/m9n7Zvaemf2ocHlZz10wrpKct5L/zW5miwB8BOCfAPQCeBfAo+7+fkkHksHMegB0unvZJ2CY2T8AGAPwW3e/pXDZvwMYdvfnCv9RXu/u/1ohY3sWwFi5t/Eu7FbUMnObcQAPA/hnlPHcBeN6BCU4b+V4Zb8LwHF3P+Hu4wB+B+ChMoyj4rn7WwCGv3bxQwD2FL7fg+knS8lljK0iuHu/ux8ofD8K4No242U9d8G4SqIcyd4K4PMZP/eisvZ7dwB/MrP9Zra73IOZRbO79xe+HwDQXM7BzIJu411KX9tmvGLOXTHbn+elD+i+6V53vx3AgwB+WHi7WpF8+m+wSqqdzmkb71KZZZvxL5Xz3BW7/Xle5Uj2PgBrZ/zcVrisIrh7X+HrEICXUXlbUQ9e20G38HWozOP5UiVt4z3bNuOogHNXzu3Py5Hs7wLYaGYdZlYL4PsAXi3DOL7BzOoKH5zAzOoAPIDK24r6VQCPFb5/DMArZRzLV1TKNt5Z24yjzOeu7Nufu3vJ/wHYhelP5D8B8G/lGEPGuNYBOFT49165xwbgJUy/rbuK6c82HgewEsCbAD4G8L8AVlTQ2P4bwBEAhzGdWC1lGtu9mH6LfhjAwcK/XeU+d8G4SnLeNF1WJBH6gE4kEUp2kUQo2UUSoWQXSYSSXSQRSnaRRCjZRRLxf8OTGL7vw01MAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# generate image by input label\n",
    "img_data = n.back_query(0)\n",
    "plt.imshow(img_data.reshape(28, 28), cmap='Greys', interpolation='None')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9579\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9561"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# add extra images by rotate existed images\n",
    "import scipy.ndimage\n",
    "print(n.correct_rate)\n",
    "for record in training_data_list:\n",
    "    all_values = record.split(',')\n",
    "    inputs = np.asfarray(all_values[1:]) / 255.0 * 0.99 + 0.01\n",
    "    inputs_rotated = scipy.ndimage.rotate(inputs.reshape(28, 28), -5, cval=0.01, reshape=False)\n",
    "    targets = np.zeros(output_nodes) + 0.01\n",
    "    targets[int(all_values[0])] = 0.99\n",
    "    n.train(inputs_rotated.flatten(), targets)\n",
    "score_card = []\n",
    "for record in test_data_list:\n",
    "    all_values = record.split(',')\n",
    "    correct_label = int(all_values[0])\n",
    "    inputs = np.asfarray(all_values[1:]) / 255.0 * 0.99 + 0.01\n",
    "    outputs = n.query(inputs)\n",
    "    # get index of max value \n",
    "    label = np.argmax(outputs)\n",
    "    if label == correct_label:\n",
    "        score_card.append(1)\n",
    "    else:\n",
    "        score_card.append(0)\n",
    "score_card_array = np.asarray(score_card)\n",
    "correct_rate = score_card_array.sum() / score_card_array.size\n",
    "correct_rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.13 ('ai')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "0cb0046d74bfb5a9fee6ebb85b9850d762e40d6b321d5d9a0aa0313eefa3c5b4"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
